# coding=euc-kr

import scrap
import re
import pdb


def unescape(s):
    s = s.replace("&lt;", "<")
    s = s.replace("&nbsp;", " ")
    s = s.replace("&gt;", ">")
    # this has to be last:
    s = s.replace("&amp;", "&")
    s = s.replace("<BR />", "\n")
    s = s.replace("<br />", "\n")
    s = s.replace("</P>", "\n")
    s = s.replace("</p>", "\n")
    return s

remove_tag_obj = re.compile(r"""<(?!img)[^<\n]*>""")
remove_empty_obj = re.compile(r"""  *\n""")

class Article:
    def __init__(self, url, id, title):
        self.url = url
        self.id = id
        self.title = title
        self.title_lower = title.lower()
        self.detail = ""
        self.detail_lower = ""
        self.keyword = ""

    def set_detail( self, detail ):
        self.detail = detail
        self.detail_lower = detail.lower()

    def __unicode__(self):
        ret = "[%s] %s\nurl: %s\nkeyword: %s\n\n%s"%(self.id, self.title, self.url, self.keyword, self.detail)
        return ret

    def __repr__(self):
        return unicode(self)

    def __cmp__(self, other):
        return self.id - other.id

class Board:
    def __init__(self, name, login_url, name_id, name_pw, board_urls, board_regex, article_regex, article_url_prefix = "", encoding="utf-8", method="POST", multiline=False):
        self.name = name
        self.login_url = login_url
        self.name_id = name_id
        self.name_pw = name_pw
        #print filter_keywords
        self.board_urls = board_urls
        self.board_regex = multiline and re.compile(board_regex.decode("euc-kr"), re.MULTILINE) or re.compile(board_regex)
        self.article_regex = multiline and re.compile(article_regex, re.MULTILINE) or re.compile(article_regex)
        self.method = method
        self.encoding = encoding
        self.last_read = 0
        self.article_url_prefix = article_url_prefix
        self.articles = []
        self.other_articles = []
        self.prev_read = None
        self.last_written_file_name = "last_written_%s.txt"%self.name
        self.filter_file_name = "filter_%s.txt"%self.name
        self.load_last_written()
        self.filter_keywords = []
        try:
            f = open(self.filter_file_name, "r")
            for filter in f.readlines():
                self.filter_keywords.append( filter.replace("\n", "").decode("euc-kr").lower() )
            f.close()
        except:
            pass
            

    def login(self, id, pw, aux={}):
        form_pairs = {self.name_id:id, self.name_pw: pw}
        for key in aux:
            form_pairs[key] = aux[key]
        return scrap.GetPage( self.login_url, form_pairs, self.encoding, self.method ) and True or False

    def get_posts(self):
        for board_url in self.board_urls:
            page = scrap.GetPage( board_url, encoding=self.encoding )
            print page.encode("euc-kr")
            iter_matchs = self.board_regex.finditer(page)
            for iter_match in iter_matchs:
                self.__process_match(iter_match)

        if self.articles:
            self.last_read = self.other_articles and max(max(self.other_articles), max(self.articles)) or max(self.articles)
        else:
            self.last_read = self.other_articles and max(self.other_articles) or self.last_read

    def __process_match(self, match_obj):
        global remove_tag_obj
        a = Article( self.article_url_prefix + match_obj.group("url"), int(match_obj.group("id")), unescape(match_obj.group("title")))
        if not self.test_newer( a ): # if newer than last
            self.other_articles.append( a )
            return
        
        page = scrap.GetPage(a.url, encoding=self.encoding)
        search_result = self.article_regex.search(page)
        if search_result:
            detail = unescape(search_result.group("detail"))
            detail, num_sub = remove_tag_obj.subn("", detail)
            detail, num_sub = remove_empty_obj.subn("", detail)
            a.set_detail( detail )
        else:
            print "detail not found"
        
        if self.filter_keywords:
            for keyword in self.filter_keywords:
                if keyword in a.title_lower or keyword in a.detail_lower:
                    a.keyword = keyword
                    self.articles.append( a )
                    return
        else:
            self.articles.append( a )
        
        self.other_articles.append( a )

    def test_newer( self, newly_read ):
        if type(self.last_read) == int:
            return self.last_read < int(newly_read.id)
        else:
            return not self.last_read or self.last_read < newly_read

    def save_last_written(self):
        if self.last_read and type(self.last_read) != int:
            f = open(self.last_written_file_name, "w")
            f.write( str(self.last_read.id) )
            f.close()

    def load_last_written(self):
        try:
            f = open(self.last_written_file_name)
            self.last_read = int(f.readlines()[0])
            f.close()
        except:
            pass

    def init_articles(self):
        self.articles = []
        self.other_articles = []
    
    def __str__(self):
        result = self.name
        for article in self.articles:
            result = "%s\n******************************************************************\n%s"%(result, article)

        return result

if __name__ == "__main__":
    cy_milal = Board("cy_milal", "http://cyxso.cyworld.nate.com/login.jsp", "email", "passwd", ["http://club.cyworld.com/club/board/general/genbrd_alllist_normal.asp?club_id=51466954&board_no=0&list_type=0&board_type=1&show_type=5"], r"""<td style="width:41px; padding:0px 0 0 0;"><span class="num">(?P<id>\d+)</span></td>\n.*\n<a class='link_list1' href='(?P<url>[^']*)'>(?P<title>[^<]*)</a>""", r"""<style type='text/css'></style><div class='club' id=main_content style='width:720px;overflow-x:auto;margin:0 0 0 0;padding:15px 0 20px 0;'><STYLE>P {MARGIN-TOP: 2px; MARGIN-BOTTOM: 2px}</STYLE>(?P<detail>(.*\n)*).*\n</div></td></tr></table></td></tr></table>""", "http://club.cyworld.com/club/board/general/", "euc-kr")
    cy_milal.login("", "", {"return_domain":"www.cyworld.com", "image1.x":"0", "image1.y":"0", "dest":"", "pop":"ok", "domain":""})#"dest":"http://minihp.cyworld.com/pims/main/pims_main.asp?tid=21234681", "jcode":"", "domain":"", "pop":"ok", "return_domain":"minihp.cyworld.com", "x":"0", "y":"0"})
    cy_milal.get_posts()
    a = unicode(max(cy_milal.articles))
    print a
